//
// See the file COPYING for redistribution information.
//

#include <config.h>
#include <cstring>
#include <map>
#include <pcrecpp.h>
#include <Naipax/Common.hpp>
#include <Naipax/Error.hpp>
#include <Naipax/Url.hpp>
#include <Naipax/LinkExtor.hpp>

using namespace std;
using namespace Naipax;

inline bool _LooksLikeOtherProtocol(const string &rUrl) {
    if (rUrl.length() < 3) {
        return false;
    }
    for (uint32_t i = 0; i < rUrl.length() - 2; ++i) {
        if (rUrl[i] == ':' && rUrl[i+1] == '/' && rUrl[i+2] == '/') {
            if (i >= 4 && string(rUrl, i-4, 4) == "http") {
                return false;
            }
            else {
                return true;
            }
        }
    }
    return false;
}

inline void _GetBaseHostUrl(const string &rUrl, string &rHostUrl) {
    uint32_t Count = 0;
    uint32_t HostUrlLen = 0;

    for (uint32_t i = 0; i < rUrl.length(); ++i) {
        if (rUrl[i] == '/') {
            ++Count;
            if (Count == 3) {
                HostUrlLen = i + 1;
                break;
            }
        }
    }
    if (Count == 3) {
        // ignore the trailing slash
        rHostUrl = string(rUrl, 0, HostUrlLen - 1);
    }
    else {
        rHostUrl = rUrl;
    }
}

LinkExtor::LinkExtor(uint32_t MaxLinks, uint32_t MaxPathDepth) {
    _mMaxLinks = MaxLinks;
    _mMaxPathDepth = MaxPathDepth;
    _mBaseUrl = "";
    _mDiscardPattern = "";
}
        
LinkExtor::~LinkExtor() {
}

void LinkExtor::Extract(const string &rHtml, vector<string> &rUrls) {
    rUrls.clear();

    // The pattern discards fragments in URLs.
    static const char sLinkPattern[] = "href=\"(.+?)(?:#.*?)?\"";
    pcrecpp::RE LinkRegexp(sLinkPattern,
                           pcrecpp::RE_Options(PCRE_UTF8|PCRE_DOTALL));
    
    pcrecpp::RE DiscardRegexp(_mDiscardPattern,
                              pcrecpp::RE_Options(PCRE_UTF8|PCRE_DOTALL));

    pcrecpp::StringPiece HtmlInput(rHtml);

    string BaseHostUrl;
    _GetBaseHostUrl(_mBaseUrl, BaseHostUrl);

    map<string, bool> LinkMap;
    string LinkStr;
    while (LinkRegexp.FindAndConsume(&HtmlInput, &LinkStr) == true) {
        if (strncmp(LinkStr.c_str(), "http://", 7) != 0
            && _LooksLikeOtherProtocol(LinkStr) == true) {
            continue;
        }
        if (strncmp(LinkStr.c_str(), "http://", 7) != 0
            && _LooksLikeOtherProtocol(LinkStr) != true) {
            if (LinkStr[0] == '/') {
                LinkStr = BaseHostUrl + LinkStr;
            }
            else {
                if (_mBaseUrl[_mBaseUrl.length() - 1] == '/') {
                    LinkStr = _mBaseUrl + LinkStr;
                }
                else {
                    LinkStr = _mBaseUrl + "/" + LinkStr;
                }
            }
        }

        Url UrlObj(LinkStr);
        if (UrlObj.IsValid() != true) {
            continue;
        }
        UrlObj.Normalize();
        string &NormalizedLink = UrlObj.AsString();

        if (NormalizedLink.length() >= Naipax::URL_SIZE) {
            continue;
        }
        if (_mDiscardPattern.length() > 0
            && DiscardRegexp.PartialMatch(NormalizedLink) == true) {
            continue;
        }
        if (UrlObj.PathDepth() > _mMaxPathDepth) {
            continue;
        }

        LinkMap[NormalizedLink] = true;
    }

    for (map<string, bool>::iterator LinkMapIt = LinkMap.begin();
         LinkMapIt != LinkMap.end(); ++LinkMapIt) {
        string rStr = (*LinkMapIt).first;
        rUrls.push_back(rStr);
    }
}
